import urllib
from bs4 import BeautifulSoup
import sqlite3

import smtplib
from email.mime.text import MIMEText
# email 用于构建邮件内容
from email.header import Header

# 用于构建邮件头

# 发信方的信息：发信邮箱，QQ 邮箱授权码
from_addr = 'xxx@qq.com'
password = '你的授权码数字'




class cnkiSpider():

    def __init__(self):
        self.main_url = 'http://xueshu.baidu.com/s?wd=%E5%9B%BE%E5%83%8F%E5%8A%A0%E5%AF%86&tn=SE_baiduxueshu_c1gjeupa&sc_f_para=sc_tasktype%3D%7BfirstSimpleSearch%7D&sc_hit=1&ie=utf-8&sort=sc_time'
        self.main_cookie = 'Ecp_notFirstLogin=XuuzD1; JSESSIONID=311B73AABFB89844EA705736DACAA490; clientId=CID1c0515083adad8128d5e8c07bcfb3d6f; Ecp_LoginStuts={"IsAutoLogin":false,"UserName":"lzlgdx","ShowName":"%E5%85%B0%E5%B7%9E%E7%90%86%E5%B7%A5%E5%A4%A7%E5%AD%A6%E5%9B%BE%E4%B9%A6%E9%A6%86","UserType":"bk","BUserName":"","BShowName":"","BUserType":"","r":"XuuzD1"}; Ecp_session=1; _pk_ses=*; KNS_SortType=; cnkiUserKey=b08fddba-c9df-a2ef-edfb-62eb6c1763bf; c_m_LinID=LinID=WEEvREcwSlJHSldRa1FhdXNzY2Z2d21iNmI3KzdxNkEyZEZSeDRjY3h0MD0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!&ot=05/26/2020 20:22:13; c_m_expire=2020-05-26 20:22:13'
        self.main_html = ''
        self.titles = []
        # 初始化链接
        self.conn = sqlite3.connect('papers.db')

    # 获取列表
    def getList(self):
        req = urllib.request.Request(self.main_url)
        # req.add_header('Cookie', self.main_cookie)
        response = urllib.request.urlopen(req)
        the_page = response.read()
        self.main_html = the_page.decode('utf8')

    # 解析列表
    def parseList(self):
        soup = BeautifulSoup(self.main_html, 'html.parser')
        all_a = soup.find_all('div', class_='sc_content')
        self.titles = []
        for i in range(len(all_a)):
            a_element = all_a[i]
            title_element = a_element.find('h3', class_='t c_font')
            title = title_element.get_text()
            # title = title.replace("\n", " ")
            print('标题:', title)

            url_element = a_element.find('a')
            url = url_element['href']
            print('链接:', url)

            bean = {
                "title": title,
                "url": url
            }
            self.titles.append(bean)

    # 获取详情
    def getDetail(self):
        for i in range(len(self.titles)):
            bean = self.titles[i]
            url = bean['url']
            req = urllib.request.Request('http:' + url)
            # req.add_header('Cookie', self.main_cookie)
            response = urllib.request.urlopen(req)
            the_page = response.read()
            html = the_page.decode('utf8')
            soup = BeautifulSoup(html, 'html.parser')
            author_element = soup.find('p', class_='author_text')
            author = author_element.get_text()
            author = author.replace("\n", " ")
            author = author.replace(" ", "")
            bean['author'] = author

            abstract_element = soup.find('p', class_='abstract')
            if abstract_element != None:
                abstract = abstract_element.get_text()
                abstract = abstract.replace("\n", " ")
                abstract = abstract.replace(" ", "")
                bean['abstract'] = abstract

    # 保存到文件
    def saveToTxt(self):
        f = open('result.txt', 'w', encoding='utf - 8')  ##ffilename可以是原来的txt文件，也可以没有然后把写入的自动创建成txt文件
        for i in range(len(self.titles)):
            bean = self.titles[i]
            f.write(bean['title'])
            f.write(bean['author'])
            if 'abstract' in bean:
                f.write('\n')
                f.write(bean['abstract'])

            f.write('\n')
        f.close()

    # 判断数据库中是否已经存在了
    def isDone(self, title):
        c = self.conn.cursor()
        cursor = c.execute("SELECT count(*) from main.papers where title = " + title)
        if cursor[0][0] == 0:
            # 不包含,保存
            c.execute("insert into main.papers value(" + title + ")")
            return False
        return True

    # 发送邮件
    def sendEmail(self):
        emil_content = ''
        for i in range(len(self.titles)):
            bean = self.titles[i]
            title = bean['title']
            has = self.isDone(title)
            if has ==False:
                emil_content = emil_content+'\n'+title

        # 发送邮件

    def start(self):
        self.getList()
        self.parseList()
        self.getDetail()
        self.saveToTxt()


if __name__ == '__main__':
    cnkiSpider().start()
